{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T10:17:08.023081Z",
     "start_time": "2020-06-20T10:17:01.701566Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T10:18:30.223719Z",
     "start_time": "2020-06-20T10:18:29.639504Z"
    }
   },
   "outputs": [],
   "source": [
    "content = pd.read_csv('qa_processed', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T10:18:39.008874Z",
     "start_time": "2020-06-20T10:18:38.965850Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>q_processed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>补发网银盾</td>\n",
       "      <td>您可以到智慧柜员机办理此项业务，方便快捷，节省您的宝贵时间哦。若去柜台办理，请刷身份证取号。</td>\n",
       "      <td>补发 网银 盾</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>代发工资</td>\n",
       "      <td>您要办理的是对公开户、销户、签约及其他业务。请刷身份证，小龙人来帮您取个号吧。</td>\n",
       "      <td>代发 工资</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>对帐单查询打印</td>\n",
       "      <td>您要办理的是对公开户、销户、签约及其他业务。请刷身份证，小龙人来帮您取个号吧。</td>\n",
       "      <td>对 帐单 查询 打印</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>理财产品取号</td>\n",
       "      <td>请说出您需要办理的业务</td>\n",
       "      <td>理财 产品 取号</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>密码修改</td>\n",
       "      <td>请您带上身份证到自助柜员机办理，简单快捷，操作容易哦。</td>\n",
       "      <td>密码 修改</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  qid question                                          answer q_processed\n",
       "0   1    补发网银盾  您可以到智慧柜员机办理此项业务，方便快捷，节省您的宝贵时间哦。若去柜台办理，请刷身份证取号。     补发 网银 盾\n",
       "1   2     代发工资         您要办理的是对公开户、销户、签约及其他业务。请刷身份证，小龙人来帮您取个号吧。       代发 工资\n",
       "2   3  对帐单查询打印         您要办理的是对公开户、销户、签约及其他业务。请刷身份证，小龙人来帮您取个号吧。  对 帐单 查询 打印\n",
       "3   4   理财产品取号                                     请说出您需要办理的业务    理财 产品 取号\n",
       "4   5     密码修改                     请您带上身份证到自助柜员机办理，简单快捷，操作容易哦。       密码 修改"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:11:59.533705Z",
     "start_time": "2020-06-20T14:11:59.328122Z"
    }
   },
   "outputs": [],
   "source": [
    "# 数据集的index（索引）还是按照正常的排序。no need\n",
    "content = content.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:12:00.852045Z",
     "start_time": "2020-06-20T14:12:00.846047Z"
    }
   },
   "outputs": [],
   "source": [
    "# from sklearn.utils import shuffle\n",
    "# shuffle(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:46:12.601352Z",
     "start_time": "2020-06-20T14:46:12.588354Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def split_to_sentence(content):\n",
    "    \"分句\"\n",
    "    pattern = re.compile(\".*?[。.？?！!]\")  # 非贪婪模式匹配文字内容\n",
    "    sents = re.findall(pattern, content)\n",
    "    return sents\n",
    "\n",
    "\n",
    "def basic_clean(content):\n",
    "    content = content.lower()\n",
    "    \n",
    "    # 去除重复字符\n",
    "    content = re.sub('(\\n)+','\\n',content)\n",
    "    content = re.sub('(\\r)+','',content)\n",
    "    content = re.sub('(\\t)+','，',content)\n",
    "\n",
    "    content = re.sub('[？]+?[！]+?','',content)\n",
    "    content = re.sub('[？]+?[。]+?','',content)\n",
    "    content = re.sub('(～)+','～',content)\n",
    "    content = re.sub('(，)+','，',content)\n",
    "    content = re.sub('(。)+','。',content)\n",
    "    content = re.sub('(～。)+','。',content)\n",
    "    content = re.sub('(？)+','？',content)\n",
    "    content = re.sub('(！)+','！',content)\n",
    "    content = re.sub('(；)+','；',content)\n",
    "    content = re.sub('( )+',' ',content).strip()\n",
    "\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:46:23.211145Z",
     "start_time": "2020-06-20T14:46:12.666354Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 我 们 去 抢 劫 吧 哎 ， 我 最 腻 味 这 种 没 有 技 术 含 量 的 活 儿 ～\n",
      " 不 理 我 就 算 了 呵 呵 ， 偶 可 不 可 以 不 回 答 这 个 问 题 啊 ， , 不 如 我 给 您 做 心 理 测 试 吧 ？\n",
      " 心 灰 意 冷 暖 暖 心\n",
      " 你 几 点 上 班 小 龙 人 二 十 四 小 时 为 您 服 务 。\n",
      " 这 就 是 天 意 天 意 难 违 啊\n",
      " 汽 车 配 置 中 的 at 是 什 么 自 动 挡 汽 车\n",
      " 能 拍 张 照 片 吗 小 龙 人 好 高 兴 ， 这 是 我 的 荣 幸\n"
     ]
    }
   ],
   "source": [
    "# 格式：一行一句话；每个字符空格隔开\n",
    "\n",
    "with open('corpus_task', 'w', encoding='utf-8') as f:\n",
    "    for idx, row in content.iterrows():\n",
    "        line = ''\n",
    "        pre = '@'\n",
    "        \n",
    "        q = basic_clean(str(row.question))\n",
    "        a = basic_clean(str(row.answer))\n",
    "        \n",
    "        q_list = split_to_sentence(q)\n",
    "        q = q_list if len(q_list) > 0 else [q]\n",
    "        \n",
    "        a_list = split_to_sentence(a)\n",
    "        a = a_list if len(a_list) > 0 else [a]        \n",
    "        \n",
    "        for qi in q:\n",
    "            for s in str(qi).lower():\n",
    "                if ord('A') <= ord(pre) <= ord('z') and  ord('A') <= ord(s) <= ord('z'):\n",
    "                    line += s\n",
    "                else:\n",
    "                    line += ' ' + s\n",
    "                pre = s\n",
    "            f.write(line + '\\n')\n",
    "        for ai in a:\n",
    "            for s in str(ai).lower():\n",
    "                if ord('A') <= ord(pre) <= ord('z') and  ord('A') <= ord(s) <= ord('z'):\n",
    "                    line += s\n",
    "                else:\n",
    "                    line += ' ' + s\n",
    "                pre = s\n",
    "            f.write(line + '\\n')\n",
    "        \n",
    "        if idx % 5000 == 0:\n",
    "            print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
